{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import random\n",
    "import numpy as np\n",
    "import torch.optim as optim\n",
    "import matplotlib.pyplot as plt\n",
    "import gym\n",
    "import torch\n",
    "import gym_uav\n",
    "from collections import deque"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Actor(nn.Module):\n",
    "    def __init__(self, n_state, n_action, n_hidden):\n",
    "        super(Actor, self).__init__()\n",
    "        self.fc1 = nn.Linear(n_state, n_hidden)\n",
    "        self.fc2 = nn.Linear(n_hidden, n_hidden)\n",
    "        self.fc3 = nn.Linear(n_hidden, n_action)\n",
    "\n",
    "    def forward(self, state):\n",
    "        x = F.relu(self.fc1(state))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        return 2 * torch.tanh(self.fc3(x)) # action = [0,1]\n",
    "\n",
    "class Critic(nn.Module):\n",
    "    def __init__(self, n_state, n_action, n_hidden):\n",
    "        super(Critic, self).__init__()\n",
    "        self.fc1 = nn.Linear(n_state+n_action, n_hidden)\n",
    "        self.fc2 = nn.Linear(n_hidden, n_hidden)\n",
    "        self.fc3 = nn.Linear(n_hidden, 1)        \n",
    "\n",
    "    def forward(self, state, action):\n",
    "        x = torch.cat((state, action), dim=1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        return self.fc3(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Buffer():\n",
    "    def __init__(self, capacity):\n",
    "        self.q = deque(maxlen=capacity)\n",
    "\n",
    "    def push_buffer(self, record):\n",
    "        self.q.append(record)\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        batch = random.sample(self.q, batch_size)\n",
    "        state, action, reward, done, next_state = zip(*batch)  \n",
    "        return state, action, reward, done, next_state\n",
    "\n",
    "    def getlen(self):\n",
    "        return len(self.q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DDPG():\n",
    "    def __init__(self, n_state, n_action, cfg):\n",
    "        self.cfg = cfg\n",
    "        self.actor = Actor(n_state, n_action, cfg.actor_hidden_dim).to(cfg.device)\n",
    "        self.critic = Critic(n_state, n_action, cfg.critic_hidden_dim).to(cfg.device)\n",
    "        self.target_actor = Actor(n_state, n_action, cfg.actor_hidden_dim).to(cfg.device)\n",
    "        self.target_critic = Critic(n_state, n_action, cfg.critic_hidden_dim).to(cfg.device)\n",
    "        self.actor_optim = optim.Adam(self.actor.parameters(), lr=cfg.lr_actor)\n",
    "        self.critic_optim = optim.Adam(self.critic.parameters(), lr=cfg.lr_critic)\n",
    "\n",
    "        for param , target_param in zip(self.actor.parameters(), self.target_actor.parameters()):\n",
    "            target_param.data.copy_(param.data)\n",
    "        for param , target_param in zip(self.critic.parameters(), self.target_critic.parameters()):\n",
    "            target_param.data.copy_(param.data)\n",
    "\n",
    "    def choose_action(self, state):\n",
    "        state = torch.FloatTensor(state).to(self.cfg.device)\n",
    "        action = self.actor(state)\n",
    "        return action.detach().cpu().numpy()\n",
    "\n",
    "    def update(self, buffer):\n",
    "        if self.cfg.batch_size > buffer.getlen():\n",
    "            return\n",
    "        state, action, reward, done, next_state = buffer.sample(self.cfg.batch_size)\n",
    "        \n",
    "        state = torch.FloatTensor(state).to(self.cfg.device)\n",
    "        action = torch.FloatTensor(action).to(self.cfg.device)\n",
    "        reward = torch.FloatTensor(reward).unsqueeze(dim=1).to(self.cfg.device)\n",
    "        done = torch.FloatTensor(done).unsqueeze(dim=1).to(self.cfg.device)\n",
    "        next_state = torch.FloatTensor(next_state).to(self.cfg.device)\n",
    "\n",
    "        next_action = self.target_actor(next_state)\n",
    "        policy_loss = -self.critic(state, self.actor(state)).mean()\n",
    "        \n",
    "        expected_value = reward + (1.0 - done) * self.cfg.discount * self.target_critic(next_state, next_action)\n",
    "        value = self.critic(state, action)\n",
    "        value_loss = torch.nn.MSELoss()(value, expected_value)\n",
    "\n",
    "        self.actor_optim.zero_grad()\n",
    "        policy_loss.backward()\n",
    "        self.actor_optim.step()\n",
    "\n",
    "        self.critic_optim.zero_grad()\n",
    "        value_loss.backward()\n",
    "        self.critic_optim.step()\n",
    "\n",
    "        # update target network\n",
    "        for param, target_param in zip(self.actor.parameters(), self.target_actor.parameters()):\n",
    "            target_param.data.copy_(\n",
    "                target_param.data * (1 - self.cfg.tau) + param.data * self.cfg.tau\n",
    "            )\n",
    "        \n",
    "        for param, target_param in zip(self.critic.parameters(), self.target_critic.parameters()):\n",
    "            target_param.data.copy_(\n",
    "                target_param.data * (1 - self.cfg.tau) + param.data * self.cfg.tau\n",
    "            )\n",
    "\n",
    "    def save(self, path):\n",
    "        torch.save(self.actor.state_dict(), path)\n",
    "\n",
    "    def load(self, path):\n",
    "        self.actor.load_state_dict(torch.load(path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OrnsteinUhlenbeckActionNoise():\n",
    "    def __init__(self, n_action):\n",
    "        self.theta = 0.15\n",
    "        self.mu = 0.0\n",
    "        self.delta = 1\n",
    "        self.sigma = 0.3\n",
    "        self.size = (n_action,)\n",
    "        self.pre_x = np.zeros(self.size)\n",
    "\n",
    "    def add_noisy(self):\n",
    "        x = self.pre_x + self.theta * (self.mu - self.pre_x) * self.delta + \\\n",
    "            self.sigma * np.random.normal(size=self.size)\n",
    "        self.pre_x = x\n",
    "        return x\n",
    "        \n",
    "    def set_action(self, action, low_bound, high_bound): # 将结果固定在action_space定义域中\n",
    "        x = self.add_noisy()\n",
    "        action += x\n",
    "        return np.clip(action, low_bound, high_bound)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config():\n",
    "    def __init__(self):\n",
    "        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "        self.ep_max = 300\n",
    "        self.buffer_capacity = 8000\n",
    "        self.batch_size = 128\n",
    "        self.actor_hidden_dim = 32\n",
    "        self.critic_hidden_dim = 64\n",
    "        self.lr_critic = 0.001\n",
    "        self.lr_actor = 0.0001\n",
    "        self.discount = 0.99\n",
    "        self.tau = 0.01 # 软更新参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func_reward(observation):\n",
    "    delta = 2.0\n",
    "    alpha = 8.0\n",
    "    beta = 25.0\n",
    "    r1 = delta * observation[9]\n",
    "    dmin = np.min(np.concatenate((observation[:9], observation[15:])))\n",
    "    r2 = -alpha * np.exp(-beta * dmin)\n",
    "    r3 = 0.1\n",
    "    r4 = -0.6 \n",
    "    return r1+r2+r3+r4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Github\\RL-UAV\\RL-UAV\\gym_uav\\envs\\uav_env.py:167: RuntimeWarning: divide by zero encountered in double_scalars\n",
      "  dist.append(final_dist / forward_dist)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n",
      "agent collides with obstacles!\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_14024/1136843207.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     16\u001b[0m         \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mddpg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchoose_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m         \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnoisy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlow\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhigh\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m         \u001b[0mnext_state\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     19\u001b[0m         \u001b[0mreward\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc_reward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnext_state\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m         \u001b[0mbuffer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpush_buffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maction\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python\\lib\\site-packages\\gym\\wrappers\\time_limit.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m     16\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_elapsed_steps\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m         ), \"Cannot call env.step() before calling reset()\"\n\u001b[1;32m---> 18\u001b[1;33m         \u001b[0mobservation\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     19\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_elapsed_steps\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_elapsed_steps\u001b[0m \u001b[1;33m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_max_episode_steps\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\Github\\RL-UAV\\RL-UAV\\gym_uav\\envs\\uav_env.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m    263\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mend_cache\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    264\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 265\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_observation\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0morient\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    266\u001b[0m         \u001b[0mnext_observation\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    267\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\Github\\RL-UAV\\RL-UAV\\gym_uav\\envs\\uav_env.py\u001b[0m in \u001b[0;36m_get_observation\u001b[1;34m(self, position, target, orient)\u001b[0m\n\u001b[0;32m    239\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m14\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mspeed\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_speed\u001b[0m \u001b[1;33m-\u001b[0m \u001b[1;36m0.5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    240\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 241\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0magent_params_pre\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0magent_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    242\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0magent_params\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'position'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mposition\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    243\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0magent_params\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'target'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python\\lib\\copy.py\u001b[0m in \u001b[0;36mdeepcopy\u001b[1;34m(x, memo, _nil)\u001b[0m\n\u001b[0;32m    144\u001b[0m     \u001b[0mcopier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_deepcopy_dispatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    145\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcopier\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 146\u001b[1;33m         \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    147\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    148\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0missubclass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python\\lib\\copy.py\u001b[0m in \u001b[0;36m_deepcopy_dict\u001b[1;34m(x, memo, deepcopy)\u001b[0m\n\u001b[0;32m    228\u001b[0m     \u001b[0mmemo\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    229\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 230\u001b[1;33m         \u001b[0my\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    231\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    232\u001b[0m \u001b[0md\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_deepcopy_dict\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python\\lib\\copy.py\u001b[0m in \u001b[0;36mdeepcopy\u001b[1;34m(x, memo, _nil)\u001b[0m\n\u001b[0;32m    144\u001b[0m     \u001b[0mcopier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_deepcopy_dispatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    145\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcopier\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 146\u001b[1;33m         \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    147\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    148\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0missubclass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python\\lib\\copy.py\u001b[0m in \u001b[0;36m_deepcopy_list\u001b[1;34m(x, memo, deepcopy)\u001b[0m\n\u001b[0;32m    203\u001b[0m     \u001b[0mappend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    204\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 205\u001b[1;33m         \u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    206\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    207\u001b[0m \u001b[0md\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_deepcopy_list\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python\\lib\\copy.py\u001b[0m in \u001b[0;36mdeepcopy\u001b[1;34m(x, memo, _nil)\u001b[0m\n\u001b[0;32m    144\u001b[0m     \u001b[0mcopier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_deepcopy_dispatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    145\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcopier\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 146\u001b[1;33m         \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    147\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    148\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0missubclass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python\\lib\\copy.py\u001b[0m in \u001b[0;36m_deepcopy_list\u001b[1;34m(x, memo, deepcopy)\u001b[0m\n\u001b[0;32m    203\u001b[0m     \u001b[0mappend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    204\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 205\u001b[1;33m         \u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    206\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    207\u001b[0m \u001b[0md\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_deepcopy_list\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python\\lib\\copy.py\u001b[0m in \u001b[0;36mdeepcopy\u001b[1;34m(x, memo, _nil)\u001b[0m\n\u001b[0;32m    151\u001b[0m             \u001b[0mcopier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"__deepcopy__\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    152\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mcopier\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 153\u001b[1;33m                 \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmemo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    154\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    155\u001b[0m                 \u001b[0mreductor\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdispatch_table\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "cfg = Config()\n",
    "print(cfg.device)\n",
    "\n",
    "env = gym.make('uav-v0')\n",
    "buffer = Buffer(cfg.buffer_capacity)\n",
    "ddpg = DDPG(env.observation_space.shape[0], env.action_space.shape[0], cfg)\n",
    "noisy = OrnsteinUhlenbeckActionNoise(env.action_space.shape[0])\n",
    "\n",
    "rewards = []\n",
    "\n",
    "for i in range(1, cfg.ep_max+1):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    ep_reward = 0\n",
    "    while done is not True:\n",
    "        action = ddpg.choose_action(state)\n",
    "        action = noisy.set_action(action, env.action_space.low[0], env.action_space.high[0])\n",
    "        next_state, _, done, _ = env.step(action)\n",
    "        reward = func_reward(next_state)\n",
    "        buffer.push_buffer((state, action, reward, done, next_state))\n",
    "        state = next_state\n",
    "        ddpg.update(buffer)\n",
    "        ep_reward += reward\n",
    "    rewards.append(ep_reward)\n",
    "    if i % 10 == 0:\n",
    "        print('epoch {}: episode reward = {}'.format(i, ep_reward))\n",
    "\n",
    "plt.plot(range(len(rewards)), rewards)\n",
    "plt.show()\n",
    "plt.savefig(\"./reward.jpg\")\n",
    "\n",
    "# 存储结果\n",
    "# import os\n",
    "# os.makedirs(\"./outputs\", exist_ok=True)\n",
    "# ddpg.save('./outputs/actor.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'d:\\\\Github\\\\RL-UAV\\\\RL-UAV'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "4ce0e62306dd6a5716965d4519ada776f947e6dfc145b604b11307c10277ef29"
  },
  "kernelspec": {
   "display_name": "Python 3.9.2 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
